using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;

using System.IO;
using System.Net;
using System.Collections;
using System.Text.RegularExpressions;
using System.Threading;
namespace WindowsFormsApp3
{
    public partial class Form1 : Form
    {
        private Hashtable urls = new Hashtable();
        private int count = 0;
        private void Crawl()
        {
            textBox2.Text += "开始爬行……\n";
            while (true)
            {
                string current = null;
                foreach (string url in urls.Keys) //找到一个还没有下載过的
                {
                    if ((bool)urls[url]) continue; //已经下載过的，不再下載
                    current = url;
                }
                if (current == null || count > 10) break;
                textBox2.Text += "爬行" + current + "页面！\n";
                string html = DownLoad(current); //下载
                urls[current] = true;
                count++;
                Parse(html); //解析，并加入新的链接
            }
            Console.WriteLine("爬行结束\n");
        }
        public string DownLoad(string url)
        {
            try
            {
                WebClient webClient = new WebClient();
                webClient.Encoding = Encoding.UTF8;
                string html = webClient.DownloadString(url);
                string fileName = count.ToString();
                File.WriteAllText(fileName, html, Encoding.UTF8);
                return html;
            }
            catch (Exception ex)
            {
                Console.WriteLine(ex.Message);
                return "";
            }
        }
        public void Parse(string html)
        {
            // 判断是不是html文本
            if (Regex.IsMatch(html, @"\<html\>[\s\S]*\<\/html\>"))
            {
            string strRef = @"(href|HREF)[]*=[]*[""'][^""'#>]+[""']";
            MatchCollection matches = new Regex(strRef).Matches(html);
            foreach (Match match in matches)
            {
                strRef = match.Value.Substring(
                    match.Value.IndexOf('=') + 1).Trim('"', '\"', '#', ',', '>');
                if (strRef.Length == 0) continue;
                if (urls[strRef] == null) urls[strRef] = false;
            }
            }
        }
        public Form1()
        {
            InitializeComponent();
        }

        private void button1_Click(object sender, EventArgs e)
        {
            urls.Add(textBox1.Text, false);
            new Thread(Crawl).Start();
        }
    }
}
